import json, os
import requests
import threading
from pyquery import PyQuery as pq
import logging
import random

_path = os.path.dirname(__file__)
logging.basicConfig(level=logging.INFO, filename=os.path.join(_path, 'log/back_ip_log.txt'))


class Check_IP(object):
    """
        通过 百度 验证抓到的免费的代理ip，ok的ip写到good_ip中，没有用的ip记到log中
    """
    def __init__(self, check_ip_file, good_ip_file):
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}
        self.check_ip_file = check_ip_file   # 保存要检查的ip的json 文件
        self.good_ip_file = good_ip_file     # 验证ok 的文件的保存位置
        self.check_ip_lists = list()  # 要检查的所有ip
        self.good_ip_lists = list()  # 验证ok的所有ip
        self.get_test_ip_lock = threading.Lock()   # 取检查ip的锁
        self.save_good_ip = threading.Lock()        # # 存验证OK ip的锁

    def read_ip(self):
        with open(self.check_ip_file, 'r', encoding='utf8') as f:
            self.check_ip_lists = json.load(fp=f)

    def write_good_ip(self):
        with open(self.good_ip_file, 'w', encoding='utf8') as f:
            json.dump(self.good_ip_lists, fp=f, ensure_ascii=False, indent=2)

    def check_ip(self):
        while True:
            if len(self.check_ip_lists) == 0:
                break
            self.get_test_ip_lock.acquire()
            ip_dict = self.check_ip_lists.pop()  # 从ip列表中取一个
            print(ip_dict)
            self.get_test_ip_lock.release()
            try:
                response = requests.get('https://www.baidu.com/', headers=self.headers, proxies=ip_dict, timeout=20)
                if response.status_code == 200:
                    print(response.status_code)
                    self.save_good_ip.acquire()
                    # self.write_good_ip(ip_dict=ip_dict)  # 把好的ip保存到一个文件中
                    self.good_ip_lists.append(ip_dict)    # 把验证好的ip保存list中
                    self.save_good_ip.release()
            except Exception as ex:
                print('ex is [{}]'.format(ex))
                self.save_good_ip.acquire()
                logging.info('{} is can not use'.format(ip_dict))
                self.save_good_ip.release()

    def main(self):
        self.read_ip()  # 从文件中读取所有的ip

        # 用100个线程来验证这些ip是否有用
        ths = []
        for t in range(100):
            t = threading.Thread(target=self.check_ip)
            ths.append(t)

        for t in ths:
            t.start()

        for t in ths:
            t.join()

        self.write_good_ip()


if __name__ == '__main__':
    check_ip_file = os.path.join(_path, 'proxy_ip/get_ip_data.json')
    good_ip_file = os.path.join(_path, 'proxy_ip/good_ip_file')

    get_ip = Check_IP(check_ip_file=check_ip_file, good_ip_file=good_ip_file)
    get_ip.main()
